# 用于针对京东的多个URL列表去重合并
import os

url_dir = "./Url_dir"
output_file = os.path.join(url_dir, "Toutiao-300.txt")

input_fd = open(os.path.join(url_dir, "Toutiao_0213_all.txt"))
url_list = input_fd.readlines()

output_fd = open(output_file, "w")
url_set = set()

for line in url_list:
    url_set.add(line)

for url in url_set:
    output_fd.write(url)

input_fd.close()
output_fd.close()
